#!/usr/bin/python
# -*- coding: UTF-8 -*-
from spider.requests_helper import Request
from lxml import html
import os


def spider(page):
    save_base_path = "G:\\novel\\"
    base_url = "http://haose103.com"
    url = base_url + "/arttype/xiaoyuanxiaoshuo.html"
    if page != 1:
        url = base_url + "/arttype/xiaoyuanxiaoshuo-{page}.html".format(page=page)

    # 获得html内容
    html_text = Request().get_html_text(url)

    # 获得xpath对象
    selector = html.fromstring(html_text)

    li_list = selector.xpath('//div[@class="box list channel"]/ul/li')
    for li in li_list:
        a_href = li.xpath('a/@href')
        url = base_url + a_href[0]
        # 获得html内容
        html_text = Request().get_html_text(url)
        # 获得xpath对象
        selector = html.fromstring(html_text)
        title = selector.xpath('//div[@class="page_title"]/text()')
        title = title[0]
        p_content = selector.xpath('//div[@class="w685"]/p/text()')
        content = selector.xpath('//div[@class="w685"]/text()')

        str = '\n'.join(p_content)
        str = str + '\n'.join(content)
        path = save_base_path + title + ".txt"

        with open(path, 'wb') as f:
            f.write(str.encode("utf-8"))

        print("下载 {title} 完成".format(title=title))


if __name__ == '__main__':
    for i in range(69):
        page = i + 1
        print("开始下载{page}页小说".format(page=page))
        spider(page)
        print("{page}页小说下载完成".format(page=page))
